package ca.uwindsor.cs.deepweb.utility.gov2;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Date;
import java.util.Vector;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.LockObtainFailedException;

import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;
import ca.uwindsor.cs.deepweb.utility.NekoHTMLHandler;

public class MultiIndexesBuilder extends FilesContentIndexer {

	protected Vector<IndexWriter> vector_IndexWirter;

	protected int indexnum;

	protected int idendswith;

	public MultiIndexesBuilder(String sourcepath, String indexdir,
			String encoding, boolean verbose, boolean create, String[] ext,
			int indexnum, int indends) {
		super(sourcepath, indexdir, encoding, verbose, create, ext);
		this.indexnum = indexnum;
		vector_IndexWirter = new Vector<IndexWriter>(indexnum);
		idendswith = indends;
	}

	/**
	 * Start to index the files in the specified directory. One file's content
	 * is an instance of Document in Lucene.
	 */
	public void generateIndex() {
		Analyzer luceneAnalyzer = new StandardAnalyzer();
		try {
			if (idendswith == -1) {
				for (int i = 0; i < indexnum; i++) {
					vector_IndexWirter.add(i, new IndexWriter(indexDir
							+ java.io.File.separator + i, luceneAnalyzer,
							isCreate));
				}
			} else {
				writer = new IndexWriter(indexDir + java.io.File.separator
						+ idendswith, luceneAnalyzer, isCreate);
			}

			long startTime = new Date().getTime();

			generateIndex(dataDir);

			long endTime = new Date().getTime();
			System.out
					.println("It takes "
							+ (endTime - startTime)
							+ " milliseconds to create index for the files in directory "
							+ dataDir.getPath()
							+ "\nTotal documnents indexed is:" + (globalID - 1));

			System.out
					.println("index is completed! \n Start to optimize the index...");
			if (idendswith == -1) {
				for (IndexWriter indexWriter : vector_IndexWirter) {
					indexWriter.optimize();
					indexWriter.close();
				}
			} else {
				writer.optimize();
				writer.close();
			}
			System.out.println("optimization is completed!");
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(-1);
		} catch (LockObtainFailedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(-1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(-1);
		}
	}

	protected void generateIndex(File directory) {
		try {
			File[] dataFiles = directory.listFiles();

			// Add documents to the index
			for (int i = 0; i < dataFiles.length; i++) {
				File file = dataFiles[i];
				if (file.isFile()) {
					if (isValidExtension(file.getName())
							&& file.length() > 2048) {
						long id = getFileNumericID(file);
						int index = new Long(id % indexnum).intValue();
						if (idendswith != -1 && index != idendswith) {
							continue;
						} else {
							// System.out.println("IDendswith=" +idendswith+"
							// Index="+ index);
							NekoHTMLHandler handler = new NekoHTMLHandler(
									encoding);
							try {
								handler.processContent(new FileInputStream(
										dataFiles[i]));
							} catch (Exception e) {
								// e.printStackTrace();
								System.err.println("Error on parsing: "
										+ dataFiles[i].getCanonicalPath());
								continue;
							}

							String content = handler.getContent().replaceAll(
									"\\s+", " ");
							if (content.length() < 2048) {
								continue;
							}

							if (isVerbose) {
								System.out.println("Indexing file "
										+ dataFiles[i].getCanonicalPath());
							}

							Document document = new Document();
							// make the ID from the path shorter - get the file
							// name
							String filepath = file.getCanonicalPath();
							int offset1 = filepath
									.lastIndexOf(java.io.File.separator);
							int offset2 = filepath.lastIndexOf(".");
							String word = new String(filepath.substring(
									offset1 + 1, offset2));

							// String word = String.valueOf(globalID);
							// System.out.println(id);

							Field FieldPath = new Field(
									FilesContentIndexer.FIELD_ID, word,
									Field.Store.YES, Field.Index.UN_TOKENIZED);

							Field FieldTitle = new Field(
									FilesContentIndexer.FIELD_TITLE, handler
											.getTitle(), Field.Store.YES,
									Field.Index.UN_TOKENIZED);

							Field FieldBody = new Field(
									FilesContentIndexer.FIELD_CONTENT, content,
									Field.Store.NO, Field.Index.TOKENIZED,
									Field.TermVector.YES);

							document.add(FieldPath);
							document.add(FieldTitle);
							document.add(FieldBody);

							// long id = getFileNumericID(file);
							// int index = new Long(id % indexnum).intValue();
							if (idendswith == -1) {
								vector_IndexWirter.get(index).addDocument(
										document);
							} else {
								writer.addDocument(document);
							}
							// txtReader.close();
							globalID++;
						}
					}
				} else if (file.isDirectory()) {
					generateIndex(dataFiles[i]);
				}
			}
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		}
	}

	protected long getFileNumericID(File file) {
		long id = Long.valueOf(file.getName().replaceAll("[^\\d]", ""));
		return id;
	}

	// protected String getContent(File file){
	//		
	// }
	//	
	// protected String getTitle(){
	//		
	// }
	//	
	// protected void initHandler(){
	// NekoHTMLHandler handler = new NekoHTMLHandler(encoding);
	// }

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// args = new String[8];
		// args[0] = "--SourceDir=G:\\gov2-corpus-extracted\\GX000\\00";
		// args[1] = "--IndexDir=F:\\indexes\\GOV2_seperated";
		// args[2] = "--encoding=latin1";
		// args[3] = "-verbose";
		// args[4]= "-create";
		// args[5]= "-resursive";
		// args[6]= "--extension=html";
		// args[7]="--IndexCount=11";

		String SourceDir = new String();
		String IndexDir = new String();
		String encoding = new String("UTF-8");
		boolean isVerbose = false;
		boolean isCreate = true;
		boolean isRecursive = false;
		int indexnum = 1;
		int idenswith = -1;

		String ext[] = new String[] { "html" };

		if (args.length < 2) {
			System.out
					.println("usage: java -jar MultiIndexesBuilder --SourceDir=sourcefiledirectory --IndexDir=indexfiledirectory --IndexCount=count --extension=txt,html,... [--idendswith=idsuffix] [--encoding=encoding](default=UTF-8) [-verbose] [-create] !");
			System.exit(0);
		} else {
			for (int i = 0; i < args.length; i++) {
				if (args[i].startsWith("--SourceDir=")) {
					SourceDir = args[i].substring(12);
				} else if (args[i].startsWith("--IndexDir=")) {
					IndexDir = args[i].substring(11);
				} else if (args[i].startsWith("--encoding=")) {
					encoding = args[i].substring(11);
				} else if (args[i].equals("-create")) {
					isVerbose = true;
				} else if (args[i].equals("-verbose")) {
					isCreate = true;
				} else if (args[i].startsWith("--extension=")) {
					String tempexts = args[i].substring(12);
					ext = tempexts.split(",");
				} else if (args[i].endsWith("-resursive")) {
					isRecursive = true;
				} else if (args[i].startsWith("--IndexCount=")) {
					indexnum = Integer.parseInt(args[i].substring(13));
				} else if (args[i].startsWith("--idendswith=")) {
					idenswith = Integer.parseInt(args[i].substring(13));
				} else {
					System.exit(-1);
				}
			}
		}

		if (SourceDir.equals("") || IndexDir.equals("")) {
			System.out
					.println("usage: java -jar MultiIndexesBuilder SourceDir=sourcefiledirectory IndexDir=indexfiledirectory  [--idendswith=idsuffix] [encoding=encoding](default=UTF-8) [-verbose]!");
			System.exit(0);
		} else {
			MultiIndexesBuilder indexer;
			// if (isRecursive) {
			// indexer = new HTMLFilesContentIndexer(SourceDir, IndexDir,
			// encoding,
			// isVerbose, isCreate, ext);
			// } else {
			// indexer = new HTMLFilesContentIndexer (SourceDir, IndexDir,
			// encoding,
			// isVerbose, isCreate, ext);
			// }
			indexer = new MultiIndexesBuilder(SourceDir, IndexDir, encoding,
					isVerbose, isCreate, ext, indexnum, idenswith);
			indexer.generateIndex();
		}

	}

}
